{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R3lwvXp168Y4"
   },
   "outputs": [],
   "source": [
    "#Downloading data and unzipping files from machine hack website\n",
    "import requests, zipfile, io\n",
    "zip_file_url = \"https://www.machinehack.com/wp-content/uploads/2020/03/MovieScriptsParticipantsData.zip\"\n",
    "r = requests.get(zip_file_url)\n",
    "z = zipfile.ZipFile(io.BytesIO(r.content))\n",
    "z.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 581
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Z-jU9Y6r7EZg",
    "outputId": "e6c6137a-3e6d-4b12-f8f3-1ece452b8e25"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/22/97/7db72a0beef1825f82188a4b923e62a146271ac2ced7928baa4d47ef2467/transformers-2.9.1-py3-none-any.whl (641kB)\n",
      "\u001b[K     |████████████████████████████████| 645kB 2.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.4)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3b/88/49e772d686088e1278766ad68a463513642a2a877487decbd691dec02955/sentencepiece-0.1.90-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 14.1MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 19.2MB/s \n",
      "\u001b[?25hCollecting tokenizers==0.7.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
      "\u001b[K     |████████████████████████████████| 3.8MB 26.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.9)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=fa84331caa899d2a1f0f9e23b258990fde7e242a66982fdea4abab161148f531\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, sacremoses, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.90 tokenizers-0.7.0 transformers-2.9.1\n"
     ]
    }
   ],
   "source": [
    "pip install transformers\n",
    "#Colab doesnt have transformers library preinstalled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "FdpfUrsa7KDL",
    "outputId": "bd540788-4f0f-4779-ab71-5b8810922eda"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import warnings;\n",
    "warnings.filterwarnings('ignore');\n",
    "pd.set_option('display.max_colwidth', 100)\n",
    "import torch\n",
    "import time\n",
    "import pickle\n",
    "import os\n",
    "from transformers import DistilBertTokenizer,DistilBertModel\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import log_loss,accuracy_score\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pEgHHsysDj-4"
   },
   "outputs": [],
   "source": [
    "seed = 123\n",
    "#Fixing seed to ensure reproducibility\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gTCu8Lld7ZPS"
   },
   "outputs": [],
   "source": [
    "cur_dir = os.getcwd()\n",
    "input_dir = '/content/MovieScriptsParticipantsData'\n",
    "script_dir = '/content/MovieScriptsParticipantsData/Scripts'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XTCalIZh74iL"
   },
   "outputs": [],
   "source": [
    "train_path = os.path.join(cur_dir,input_dir,'Train.csv')\n",
    "test_path = os.path.join(cur_dir,input_dir,'Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-yqLrkNK76vL"
   },
   "outputs": [],
   "source": [
    "df_raw = pd.read_csv(train_path)\n",
    "df_test = pd.read_csv(test_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "id": "Zi8WRkbvElEj",
    "outputId": "402a20c8-9117-45ee-cc91-eeef01741f6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1978 entries, 0 to 1977\n",
      "Data columns (total 2 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   File_Name  1978 non-null   object\n",
      " 1   Labels     1978 non-null   int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 31.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df_raw.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "id": "O09GU5VBEnOp",
    "outputId": "7fd9e755-7d7c-4cb9-cc35-1dd51c0e421f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 849 entries, 0 to 848\n",
      "Data columns (total 1 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   File_Name  849 non-null    object\n",
      "dtypes: object(1)\n",
      "memory usage: 6.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Bx1VRMi27_FS",
    "outputId": "d435f89a-7265-4b5e-d52f-a4236deae687"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2827"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['Labels'] = -1 \n",
    "df_tot = pd.concat([df_raw,df_test], axis = 0)\n",
    "#Concatenatig both train and test sets for easy processing\n",
    "df_tot.reset_index(inplace= True,drop=True)\n",
    "len(df_tot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "colab_type": "code",
    "id": "WGlyOIji9tsj",
    "outputId": "424708bb-f8b4-4d81-8637-eb1cfcf1e284"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2827 entries, 0 to 2826\n",
      "Data columns (total 2 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   File_Name  2827 non-null   object\n",
      " 1   Labels     2827 non-null   int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 44.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df_tot.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 164,
     "referenced_widgets": [
      "2f6c247da91440489b340aefe64cfe57",
      "bdcafd7f4c0842d19302cb7a9175149d",
      "df6540e88ad446e399962ffc5699a21a",
      "4397afd8a0f34d159fb73ccae33b5a51",
      "a7eef77466bc407790eb31b2b7005d7a",
      "e8a5e1f20dee403e8c251183f9ccec96",
      "1e80840242c849bfa4be15e68605f77d",
      "d85c9a701cef4ad79508ced6c06c7f8c",
      "3f8b0bde679d44dbb9b65bdedb7ce978",
      "7ef97b2a49aa4f01af99089138605ccd",
      "c0962556e54448b2be92db59d43d8b5e",
      "7ea44e58b53041ae823c8840ab659aee",
      "042e33b9a2f343cd897715b66d3400e5",
      "7798dbdeb9084f09995c68839b99bfcb",
      "47660ec6de4440fdb481604025a4475a",
      "bbb83979703543e1b79fa11ea048acc7",
      "fef3137b851c4e5fbb64f526ea60b48f",
      "2b9ab4ce2c1f44a1a120127e257f85bd",
      "435ae68cb1b14688810a88c305c98252",
      "8bd3f184746c46ecb4379e13a201d095",
      "d2a7ab63dfbe483eb6f088dad17b9572",
      "d1ffe11d54b14908afe65f60be8af068",
      "7bff275064b74963bf16cb89b0c5f0da",
      "c2f81c251a80499c9f5f6fd57de4d62e"
     ]
    },
    "colab_type": "code",
    "id": "mywYzK4m8JCL",
    "outputId": "d455992a-81da-433e-b6ea-0db9474bb0c1"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f6c247da91440489b340aefe64cfe57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f8b0bde679d44dbb9b65bdedb7ce978",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef3137b851c4e5fbb64f526ea60b48f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n",
    "# Initializing preinitialized distilbert tokenizer and model. Distilbert chosen based on memory and speed criteria."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Dwpi0-V08qqT"
   },
   "outputs": [],
   "source": [
    "#Reading scripts from the files\n",
    "def script_reader(file_name):\n",
    "    script_path = os.path.join(cur_dir,script_dir,file_name)\n",
    "    with open (script_path, \"r\") as myfile:\n",
    "        script =  myfile.read()\n",
    "        return script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9_Muk1ov8zLz"
   },
   "outputs": [],
   "source": [
    "#Each script at a time is converted into a datset and processed\n",
    "class ScriptDataset(Dataset):\n",
    "    def __init__(self, file_path,tokenizer,seq_len = 510):\n",
    "        #Sequence length = Maximum possible sequence length(512) which can be fed to model - start and end tokens(2) \n",
    "        self.file_path =  file_path\n",
    "        self.script = script_reader(self.file_path)\n",
    "        #Each script is loaded only when it is being processed and deleted once tokenization and encoding is done\n",
    "        self.tokenizer = tokenizer\n",
    "        self.seq_len = seq_len\n",
    "        self.tokens = self.tokenizer.tokenize(self.script)\n",
    "        del self.script\n",
    "        self.sequences = [self.tokens[i:i+self.seq_len] for i in range(0,len(self.tokens),self.seq_len)][:-1]\n",
    "        #Drops last sequence of unequal length\n",
    "        del self.tokens\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.sequences) \n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return torch.tensor(self.tokenizer.encode(self.sequences[idx], add_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6_HlO4409B51"
   },
   "outputs": [],
   "source": [
    "#Dividing a script into sequence and creating a embedding for each sequence and concatening all those sequence embeddings\n",
    "def create_bert_embedding(file_path,tokenizer,model,batch_size = 2):\n",
    "    model = model.to(device)\n",
    "    model = model.eval()\n",
    "    \n",
    "    script_set = ScriptDataset(file_path,tokenizer)\n",
    "    batch_loader = DataLoader(script_set,batch_size = batch_size, shuffle = False,drop_last=False)\n",
    "    del script_set\n",
    "    \n",
    "    seq_embeddings = []\n",
    "    for i,batch in enumerate(batch_loader):\n",
    "        batch = batch.to(device)\n",
    "        #Avoid gradient calculation , for speed and memory issues\n",
    "        with torch.no_grad():\n",
    "            seq_embed = model(batch)[0][:,0,:]\n",
    "            #Collecting only the final hidden state corresponding to CLS token which can be used as sequence embedding(imperfect but still works)\n",
    "        del batch\n",
    "        seq_embeddings.append(seq_embed)\n",
    "        del seq_embed\n",
    "    del batch_loader\n",
    "           \n",
    "    return torch.cat(seq_embeddings,dim = 0).cpu().detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qZIOEvnP90P-"
   },
   "outputs": [],
   "source": [
    "tik = time.time()\n",
    "Embeddings = []\n",
    "for i in range(len(df_tot)):\n",
    "    Embeddings.append(create_bert_embedding(df_tot.File_Name[i],tokenizer,model))\n",
    "    if i % 50 == 0:\n",
    "        print(i)\n",
    "\n",
    "print('Embedding completed')\n",
    "\n",
    "with open('/content/drive/My Drive/Script_Classification/embeddings.pickle', 'wb') as handle:\n",
    "    pickle.dump(Embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "#Saving the embeddings in the drive incase the colab session gets terminated \n",
    "print('Pickling completed')\n",
    "\n",
    "\n",
    "df_tot['Embeds'] = Embeddings\n",
    "tok = time.time()\n",
    "print (f'{round(tok-tik,2)}s')\n",
    "\n",
    "with open('/content/drive/My Drive/Script_Classification/df_total.pickle', 'wb') as handle:\n",
    "    pickle.dump(df_tot, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "#Saving the dataframe so that further analysis can start from this point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7oDeL9AmD80T"
   },
   "outputs": [],
   "source": [
    "df_tot_path = \"/content/drive/My Drive/Script_Classification/df_total.pickle\"\n",
    "\n",
    "with open(df_tot_path, 'rb') as handle:\n",
    "    df_tot = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "oCg-w6zzGX53",
    "outputId": "7e789b62-888d-4040-9f7d-2bf2186b4d5b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_Name</th>\n",
       "      <th>Labels</th>\n",
       "      <th>Embeds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2822</th>\n",
       "      <td>file_2474.txt</td>\n",
       "      <td>-1</td>\n",
       "      <td>[[-0.1262939, -0.019628199, -0.005916388, -0.16452256, 0.09194111, -0.08190768, 0.009038988, 0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823</th>\n",
       "      <td>file_863.txt</td>\n",
       "      <td>-1</td>\n",
       "      <td>[[-0.16330343, -0.0963432, 0.08022924, 0.01302716, 0.13003008, -0.18599892, 0.19946839, 0.130659...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2824</th>\n",
       "      <td>file_1547.txt</td>\n",
       "      <td>-1</td>\n",
       "      <td>[[0.0355055, -0.112331845, 0.09806109, -0.035105072, -0.005587622, -0.42070362, 0.14120506, 0.29...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2825</th>\n",
       "      <td>file_1292.txt</td>\n",
       "      <td>-1</td>\n",
       "      <td>[[0.11228424, -0.20076993, -0.0851114, -0.1541715, -0.0011653934, -0.10996413, 0.182178, 0.09550...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2826</th>\n",
       "      <td>file_1910.txt</td>\n",
       "      <td>-1</td>\n",
       "      <td>[[-0.0010850747, -0.053630237, 0.17418769, -0.12784621, 0.1681908, -0.33785436, 0.052511718, 0.1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          File_Name  ...                                                                                               Embeds\n",
       "2822  file_2474.txt  ...  [[-0.1262939, -0.019628199, -0.005916388, -0.16452256, 0.09194111, -0.08190768, 0.009038988, 0.1...\n",
       "2823   file_863.txt  ...  [[-0.16330343, -0.0963432, 0.08022924, 0.01302716, 0.13003008, -0.18599892, 0.19946839, 0.130659...\n",
       "2824  file_1547.txt  ...  [[0.0355055, -0.112331845, 0.09806109, -0.035105072, -0.005587622, -0.42070362, 0.14120506, 0.29...\n",
       "2825  file_1292.txt  ...  [[0.11228424, -0.20076993, -0.0851114, -0.1541715, -0.0011653934, -0.10996413, 0.182178, 0.09550...\n",
       "2826  file_1910.txt  ...  [[-0.0010850747, -0.053630237, 0.17418769, -0.12784621, 0.1681908, -0.33785436, 0.052511718, 0.1...\n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tot.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "vtIK8zCs_Icp",
    "outputId": "1b926935-27c9-4246-9c87-8f8eba584289"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1978, 849)"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw = df_tot[df_tot.Labels != -1]\n",
    "df_test = df_tot[df_tot.Labels == -1]\n",
    "df_test.reset_index(inplace= True,drop=True)\n",
    "len(df_raw),len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 408
    },
    "colab_type": "code",
    "id": "WaWwD3Ao_cBE",
    "outputId": "881d8c94-af71-4c20-e9fa-d8b665276fcd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6     405\n",
       "19    261\n",
       "4     243\n",
       "0     203\n",
       "5     141\n",
       "15    134\n",
       "1     116\n",
       "16    109\n",
       "11    104\n",
       "8      79\n",
       "14     75\n",
       "7      27\n",
       "2      25\n",
       "20     18\n",
       "13     15\n",
       "21      9\n",
       "12      4\n",
       "9       3\n",
       "3       2\n",
       "17      2\n",
       "10      2\n",
       "18      1\n",
       "Name: Labels, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.Labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 408
    },
    "colab_type": "code",
    "id": "u7cD1mby_vCz",
    "outputId": "e28f0142-9c64-4525-f128-c0d7da893c90"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6     405\n",
       "19    261\n",
       "4     243\n",
       "0     203\n",
       "5     141\n",
       "15    134\n",
       "1     116\n",
       "16    109\n",
       "11    104\n",
       "8      79\n",
       "14     75\n",
       "7      27\n",
       "2      25\n",
       "12     20\n",
       "20     18\n",
       "13     15\n",
       "9      15\n",
       "3      10\n",
       "10     10\n",
       "17     10\n",
       "21      9\n",
       "18      5\n",
       "Name: Labels, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For 5 fold stratified cross validation, the value counts for each label must be atleast 5\n",
    "df_rare = df_raw[df_raw.Labels.apply(lambda x: x in [12,9,3,17,10,18])]\n",
    "df_new = pd.concat([df_raw] + [df_rare]*4)\n",
    "df_new.reset_index(inplace= True,drop=True)\n",
    "\n",
    "df_new.Labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mp-pAIUTAMGg"
   },
   "outputs": [],
   "source": [
    "#Creating a new column and assigning each row into a fold\n",
    "df_new[\"kfold\"] = -1\n",
    "skf = StratifiedKFold()\n",
    "for fold, (train_idx, val_idx) in enumerate(skf.split(X=df_new, y=df_new['Labels'].values)):\n",
    "                    df_new.loc[val_idx, 'kfold'] = fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rJ_VuPaZBcdP"
   },
   "outputs": [],
   "source": [
    "#Create a new dataframe with array values\n",
    "def create_df_embeds(feat_array, df):\n",
    "    cols_num = feat_array[0].shape[0]\n",
    "    col_names = ['feat_' + str(i+1) for i in range(cols_num)]\n",
    "    df_feat = pd.DataFrame(data = feat_array,columns = col_names)\n",
    "    df_fit = pd.concat([df, df_feat], axis=1)\n",
    "    return df_fit,col_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HlEJrB_WCLjJ"
   },
   "outputs": [],
   "source": [
    "def train(df_fit,classifier,feat_cols,label_col = 'Labels'):\n",
    "    \n",
    "    folds = df_fit.kfold.nunique()\n",
    "    FOLD_MAPPPING = {fold:list(set(range(folds)) - set([fold]))  for fold in range(folds)}\n",
    "    clfs = [classifier]*folds\n",
    "    \n",
    "    train_losses = []\n",
    "    train_accuracies = []\n",
    "    valid_losses = []\n",
    "    valid_accuracies = []\n",
    "    \n",
    "    for FOLD in range(5):\n",
    "        \n",
    "        train_df = df_fit[df_fit.kfold.isin(FOLD_MAPPPING.get(FOLD))].reset_index(drop=True)\n",
    "        valid_df = df_fit[df_fit.kfold==FOLD].reset_index(drop=True)\n",
    "        \n",
    "        clf = clfs[FOLD]\n",
    "        clf.fit(train_df[feat_cols].values, train_df[label_col])\n",
    "        \n",
    "        train_probs = clf.predict_proba(train_df[feat_cols].values)\n",
    "        train_preds = clf.predict(train_df[feat_cols].values)\n",
    "        \n",
    "        train_accuracy = accuracy_score(train_df[label_col], train_preds)\n",
    "        train_loss = log_loss(train_df[label_col], train_probs)\n",
    "        \n",
    "        train_losses.append(train_loss)\n",
    "        train_accuracies.append(train_accuracy)\n",
    "\n",
    "        valid_probs = clf.predict_proba(valid_df[feat_cols].values)\n",
    "        valid_preds = clf.predict(valid_df[feat_cols].values)\n",
    "        \n",
    "        valid_accuracy = accuracy_score(valid_df[label_col], valid_preds)\n",
    "        valid_loss = log_loss(valid_df[label_col], valid_probs)\n",
    "        \n",
    "        valid_losses.append(valid_loss)\n",
    "        valid_accuracies.append(valid_accuracy)\n",
    "        \n",
    "    mean_train_loss,std_train_loss = np.mean(train_losses),np.std(train_losses)\n",
    "    mean_train_accuracy,std_train_accuracy  = np.mean(train_accuracies),np.std(train_accuracies)\n",
    "\n",
    "    mean_valid_loss,std_valid_loss = np.mean(valid_losses),np.std(valid_losses)\n",
    "    mean_valid_accuracy,std_valid_accuracy  = np.mean(valid_accuracies),np.std(valid_accuracies)\n",
    "\n",
    "    print(f'train loss = {mean_train_loss} with {np.round((std_train_loss/mean_train_loss)*100, 2)}% error')\n",
    "    print(f'valid loss = {mean_valid_loss} with {np.round((std_valid_loss/mean_valid_loss)*100, 2)}% error')\n",
    "\n",
    "    print(f'train accuracy = {mean_train_accuracy} with {np.round((std_train_accuracy/mean_train_accuracy)*100, 2)}% error')\n",
    "    print(f'valid accuracy = {mean_valid_accuracy} with {np.round((std_valid_accuracy/mean_valid_accuracy)*100, 2)}% error')\n",
    "    \n",
    "    return clfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZRUoQwTkCzAU"
   },
   "outputs": [],
   "source": [
    "def predict(df,clfs,feat_cols,identifier = ['File_Name']):\n",
    "    df_fit = df[feat_cols]\n",
    "    num_targets = len(clfs[0].classes_)\n",
    "    for i,clf in enumerate(clfs):\n",
    "        probs = clf.predict_proba(df_fit.values)\n",
    "        if i == 0:\n",
    "            predictions = probs\n",
    "        else:\n",
    "            predictions += probs\n",
    "    \n",
    "    predictions /= len(clfs)\n",
    "    \n",
    "    pred_df = pd.DataFrame(predictions,columns=list(range(num_targets)))\n",
    "    df_subm = pd.concat([df[identifier], pred_df], axis=1)\n",
    "       \n",
    "    return df_subm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "j4ENLS43DII_",
    "outputId": "5c15fac7-056f-40ec-82eb-81e144d75f9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss = 1.6917428895320423 with 1.45% error\n",
      "valid loss = 2.0303416478260976 with 1.25% error\n",
      "train accuracy = 0.3731562288108078 with 2.79% error\n",
      "valid accuracy = 0.2630336113094734 with 5.09% error\n"
     ]
    }
   ],
   "source": [
    "mean_array = np.stack((df_new.Embeds.apply(lambda x: x.mean(axis = 0)).values),axis = 0)\n",
    "df_mean,cols = create_df_embeds(mean_array, df_new)\n",
    "classif = LogisticRegression(random_state=seed)\n",
    "LR_clfs = train(df_mean,classif,cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 224
    },
    "colab_type": "code",
    "id": "HHKjzIayDY3t",
    "outputId": "8026bd1c-6526-4822-c640-0ad6d4293b1f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File_Name</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>file_2300.txt</td>\n",
       "      <td>0.001829</td>\n",
       "      <td>0.008076</td>\n",
       "      <td>0.002059</td>\n",
       "      <td>0.004352</td>\n",
       "      <td>0.146735</td>\n",
       "      <td>0.013130</td>\n",
       "      <td>0.286642</td>\n",
       "      <td>0.007446</td>\n",
       "      <td>0.010214</td>\n",
       "      <td>0.004847</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>0.194803</td>\n",
       "      <td>0.000360</td>\n",
       "      <td>0.001027</td>\n",
       "      <td>0.020801</td>\n",
       "      <td>0.058224</td>\n",
       "      <td>0.017925</td>\n",
       "      <td>0.002229</td>\n",
       "      <td>0.000405</td>\n",
       "      <td>0.217374</td>\n",
       "      <td>0.000220</td>\n",
       "      <td>0.001242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>file_809.txt</td>\n",
       "      <td>0.002700</td>\n",
       "      <td>0.004175</td>\n",
       "      <td>0.000680</td>\n",
       "      <td>0.001996</td>\n",
       "      <td>0.710791</td>\n",
       "      <td>0.017324</td>\n",
       "      <td>0.073842</td>\n",
       "      <td>0.001144</td>\n",
       "      <td>0.006624</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.000303</td>\n",
       "      <td>0.000582</td>\n",
       "      <td>0.000702</td>\n",
       "      <td>0.000973</td>\n",
       "      <td>0.155835</td>\n",
       "      <td>0.002312</td>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.000377</td>\n",
       "      <td>0.015853</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>file_1383.txt</td>\n",
       "      <td>0.095004</td>\n",
       "      <td>0.333355</td>\n",
       "      <td>0.003540</td>\n",
       "      <td>0.000359</td>\n",
       "      <td>0.004653</td>\n",
       "      <td>0.000106</td>\n",
       "      <td>0.053628</td>\n",
       "      <td>0.003291</td>\n",
       "      <td>0.019389</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.005490</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.001753</td>\n",
       "      <td>0.000659</td>\n",
       "      <td>0.368461</td>\n",
       "      <td>0.002111</td>\n",
       "      <td>0.001176</td>\n",
       "      <td>0.098888</td>\n",
       "      <td>0.007732</td>\n",
       "      <td>0.000181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>file_983.txt</td>\n",
       "      <td>0.011831</td>\n",
       "      <td>0.022818</td>\n",
       "      <td>0.001852</td>\n",
       "      <td>0.003467</td>\n",
       "      <td>0.163525</td>\n",
       "      <td>0.149875</td>\n",
       "      <td>0.274547</td>\n",
       "      <td>0.001712</td>\n",
       "      <td>0.010987</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.000202</td>\n",
       "      <td>0.000755</td>\n",
       "      <td>0.001996</td>\n",
       "      <td>0.002593</td>\n",
       "      <td>0.012378</td>\n",
       "      <td>0.254647</td>\n",
       "      <td>0.009736</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>0.000701</td>\n",
       "      <td>0.062396</td>\n",
       "      <td>0.000768</td>\n",
       "      <td>0.000590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>file_1713.txt</td>\n",
       "      <td>0.198234</td>\n",
       "      <td>0.257614</td>\n",
       "      <td>0.043991</td>\n",
       "      <td>0.008591</td>\n",
       "      <td>0.026813</td>\n",
       "      <td>0.012197</td>\n",
       "      <td>0.080509</td>\n",
       "      <td>0.037501</td>\n",
       "      <td>0.064655</td>\n",
       "      <td>0.001037</td>\n",
       "      <td>0.000221</td>\n",
       "      <td>0.079341</td>\n",
       "      <td>0.000895</td>\n",
       "      <td>0.021821</td>\n",
       "      <td>0.044294</td>\n",
       "      <td>0.019674</td>\n",
       "      <td>0.046538</td>\n",
       "      <td>0.002853</td>\n",
       "      <td>0.002131</td>\n",
       "      <td>0.038443</td>\n",
       "      <td>0.007280</td>\n",
       "      <td>0.005366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       File_Name         0         1  ...        19        20        21\n",
       "0  file_2300.txt  0.001829  0.008076  ...  0.217374  0.000220  0.001242\n",
       "1   file_809.txt  0.002700  0.004175  ...  0.015853  0.000085  0.000209\n",
       "2  file_1383.txt  0.095004  0.333355  ...  0.098888  0.007732  0.000181\n",
       "3   file_983.txt  0.011831  0.022818  ...  0.062396  0.000768  0.000590\n",
       "4  file_1713.txt  0.198234  0.257614  ...  0.038443  0.007280  0.005366\n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_mean_array = np.stack((df_test.Embeds.apply(lambda x: x.mean(axis = 0)).values),axis = 0)\n",
    "df_test_mean,cols = create_df_embeds(test_mean_array, df_test)\n",
    "df_submission = predict(df_test_mean,LR_clfs,cols)\n",
    "df_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uZB52PMXHHGf"
   },
   "outputs": [],
   "source": [
    "def create_file(df,file_name):\n",
    "    file_path = os.path.join(cur_dir,(file_name + '.xlsx'))\n",
    "    df.to_excel(file_path, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IarHRMe1HYYr"
   },
   "outputs": [],
   "source": [
    "create_file(df_submission,'bert_mean')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Machine hack submission.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "042e33b9a2f343cd897715b66d3400e5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "1e80840242c849bfa4be15e68605f77d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2b9ab4ce2c1f44a1a120127e257f85bd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2f6c247da91440489b340aefe64cfe57": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_df6540e88ad446e399962ffc5699a21a",
       "IPY_MODEL_4397afd8a0f34d159fb73ccae33b5a51"
      ],
      "layout": "IPY_MODEL_bdcafd7f4c0842d19302cb7a9175149d"
     }
    },
    "3f8b0bde679d44dbb9b65bdedb7ce978": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c0962556e54448b2be92db59d43d8b5e",
       "IPY_MODEL_7ea44e58b53041ae823c8840ab659aee"
      ],
      "layout": "IPY_MODEL_7ef97b2a49aa4f01af99089138605ccd"
     }
    },
    "435ae68cb1b14688810a88c305c98252": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d1ffe11d54b14908afe65f60be8af068",
      "max": 267967963,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d2a7ab63dfbe483eb6f088dad17b9572",
      "value": 267967963
     }
    },
    "4397afd8a0f34d159fb73ccae33b5a51": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d85c9a701cef4ad79508ced6c06c7f8c",
      "placeholder": "​",
      "style": "IPY_MODEL_1e80840242c849bfa4be15e68605f77d",
      "value": " 232k/232k [00:00&lt;00:00, 296kB/s]"
     }
    },
    "47660ec6de4440fdb481604025a4475a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7798dbdeb9084f09995c68839b99bfcb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7bff275064b74963bf16cb89b0c5f0da": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7ea44e58b53041ae823c8840ab659aee": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bbb83979703543e1b79fa11ea048acc7",
      "placeholder": "​",
      "style": "IPY_MODEL_47660ec6de4440fdb481604025a4475a",
      "value": " 442/442 [00:06&lt;00:00, 67.7B/s]"
     }
    },
    "7ef97b2a49aa4f01af99089138605ccd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8bd3f184746c46ecb4379e13a201d095": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c2f81c251a80499c9f5f6fd57de4d62e",
      "placeholder": "​",
      "style": "IPY_MODEL_7bff275064b74963bf16cb89b0c5f0da",
      "value": " 268M/268M [00:06&lt;00:00, 44.3MB/s]"
     }
    },
    "a7eef77466bc407790eb31b2b7005d7a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "bbb83979703543e1b79fa11ea048acc7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bdcafd7f4c0842d19302cb7a9175149d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c0962556e54448b2be92db59d43d8b5e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7798dbdeb9084f09995c68839b99bfcb",
      "max": 442,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_042e33b9a2f343cd897715b66d3400e5",
      "value": 442
     }
    },
    "c2f81c251a80499c9f5f6fd57de4d62e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d1ffe11d54b14908afe65f60be8af068": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d2a7ab63dfbe483eb6f088dad17b9572": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "d85c9a701cef4ad79508ced6c06c7f8c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "df6540e88ad446e399962ffc5699a21a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e8a5e1f20dee403e8c251183f9ccec96",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a7eef77466bc407790eb31b2b7005d7a",
      "value": 231508
     }
    },
    "e8a5e1f20dee403e8c251183f9ccec96": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fef3137b851c4e5fbb64f526ea60b48f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_435ae68cb1b14688810a88c305c98252",
       "IPY_MODEL_8bd3f184746c46ecb4379e13a201d095"
      ],
      "layout": "IPY_MODEL_2b9ab4ce2c1f44a1a120127e257f85bd"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
